﻿using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using System.Net;
using System.Collections;
using System.Text.RegularExpressions;
using System.Threading;
using static System.Net.WebRequestMethods;
using System.IO.Pipes;
using System.Security.Cryptography.X509Certificates;
using WallpaperDemo.Models;

namespace WallpaperCrawler
{
    public class Crawler
    {
        public Hashtable urls = new Hashtable();
        //public Hashtable imgsComplete = new Hashtable();
        public Queue<string> pending = new Queue<string>();
        private int count = 0;
        ////private int imgcount = 0;
        ////public int numOfPics = 1;

        //public delegate void CrawlerUrlEventHandler(UrlInfo urlInfo);
        //public virtual event CrawlerUrlEventHandler CrawUrlEvent;
        //URL检测表达式，用于在HTML文本中查找URL
        public static readonly string UrlDetectRegex = @"(href|HREF)\s*=\s*[""'](?<url>[^""'#>]+)[""']";
        //URL解析表达式
        public static readonly string urlParseRegex = @"^(?<site>(?<protocal>https?)://(?<host>[\w\d.-]+)(:\d+)?($|/))(\w+/)*(?<file>[^#?]*)";

        public static readonly string ImagesRegex = @"(srcSet)\s*=\s*[""'](?<url>[^""'#>\,\s]+)";

        public static readonly string ImgPraseRegex = @"https?://(?<host>[\w\d.-]+)/photo-(?<id1>\d+)-(?<id2>\w+).+(q=(\d+))$";
        public string HostFilter { get; set; }
        public static string FileFilter = ".(html?|aspx|jsp|php)$|^[^.]*$";

        ////public void Run(int num, out Crawler crawler)
        /// /{
            ////Crawler myCrawler = new Crawler();

            ////myCrawler.numOfPics = num;

            ////string startUrl = "https://www.unsplash.com";
            /*
            if (args.Length >= 1 && args != null)
            {
                startUrl = args;
                if (!Regex.IsMatch(startUrl, @"://")) startUrl = "https://" + startUrl;
            }
            */
            ////Match m = Regex.Match(startUrl, urlParseRegex);
            ////string s = m.Groups["host"].Value;
            //s = s.Split('.', 2)[1];
            ////myCrawler.HostFilter = s + "$";

            ////myCrawler.urls.Add(startUrl, false);
            ////myCrawler.pending.Enqueue(startUrl);
            ////crawler = myCrawler;

            ////new Thread(myCrawler.Crawl).Start();
        ////}

        public async void Crawl()
        {
            while (count < 255)
            {
                if (pending.Count == 0) continue;
                string url = pending.Dequeue();
                if (urls[url] != null)
                {
                    if ((bool)urls[url]) { continue; }
                }
                    
                if (url == null) break;
                if (url == "") break;
                //new Thread(() => this.Crawl(url, count)).Start();
                this.Crawl(url, count);
                count++;
            }
        }

        public void Crawl(string url, int cc)
        {
            string html = DownLoad(url, cc);
            Prase(url, html);
        }

        public string FixUrl(string url, string pageUrl)
        {
            if (url.Contains("://"))
            { //完整路径
                return url;
            }
            if (url.StartsWith("//"))
            {
                Match urlMatch = Regex.Match(pageUrl, urlParseRegex);
                string protocal = urlMatch.Groups["protocal"].Value;
                return protocal + ":" + url;
            }
            if (url.StartsWith("/"))
            {
                Match urlMatch = Regex.Match(pageUrl, urlParseRegex);
                String site = urlMatch.Groups["site"].Value;
                return site.EndsWith("/") ? site + url.Substring(1) : site + url;
            }

            if (url.StartsWith("../"))
            {
                url = url.Substring(3);
                int idx = pageUrl.LastIndexOf('/');
                return FixUrl(url, pageUrl.Substring(0, idx));
            }

            if (url.StartsWith("./"))
            {
                return FixUrl(url.Substring(2), pageUrl);
            }
            //非上述开头的相对路径
            int end = pageUrl.LastIndexOf("/");
            return pageUrl.Substring(0, end) + "/" + url;
        }

        public string DownLoad(string url, int cc)
        {
            string html = "";
            //string state = "";
            //string e = "";
            //try
            //{
                HttpWebRequest Myrq = WebRequest.Create(url) as HttpWebRequest;
                Myrq.KeepAlive = false;//持续连接
                Myrq.Timeout = 30 * 1000;//30秒，*1000是因为基础单位为毫秒
                Myrq.Method = "GET";//请求方法
                Myrq.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3";//自己去network里面找
                Myrq.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/113.0";

                //接受返回
                HttpWebResponse Myrp = (HttpWebResponse)Myrq.GetResponse();
                if (Myrp.StatusCode != HttpStatusCode.OK)
                { return html; }

                using (FileStream fl = new FileStream(cc.ToString(), FileMode.Create))//展开一个流
                {
                    Myrp.GetResponseStream().CopyTo(fl);//复制到当前文件夹
                    
                }
                html = System.IO.File.ReadAllText(cc.ToString());

                //string fileName = cc.ToString()+"!";
                //System.IO.File.WriteAllText(fileName, html, Encoding.UTF8);
                
                //state = "已爬取！";
            //}
            //catch (Exception ex)
            //{
            //    state = "失败...";
            //    e = ex.Message;
            //}
            
            urls[url] = true;
            //UrlInfo urlInfo = new UrlInfo(url, state, e);
            //CrawUrlEvent(urlInfo);
            return html;
        }

        public bool ifSkip(string url)
        {
            if (url == null || url == "" || url.Contains("javascript:")) return true;

            Match linkUrlMatch = Regex.Match(url, urlParseRegex);

            string host = linkUrlMatch.Groups["host"].Value;
            string file = linkUrlMatch.Groups["file"].Value;
            

            if (!Regex.IsMatch(host, HostFilter)) return true;
            if (!Regex.IsMatch(file, FileFilter)) return true;
            return false;
        }

        public void DownloadPic(string url, string name)
        {
            WebClient client = new WebClient();
            try
            {
                client.DownloadFile(url, name);
            } catch (Exception e)
            {
                return;
            }
            
        }

        public void Prase(string url, string html)
        {
            string other;
            
            MatchCollection matches = new Regex(UrlDetectRegex).Matches(html);
            /*
            MatchCollection imgs = new Regex(ImagesRegex).Matches(html);
            //client.DownloadFile(imgs[0].Groups["url"].Value, "head.jpg");
            int a = 0;
            foreach(Match img in imgs)
            {
                if (a >= numOfPics) break;
                string imgurl = img.Groups["url"].Value;
                Match imgMatch = Regex.Match(imgurl, ImgPraseRegex);
                string name = imgMatch.Groups["id1"].Value;
                if (name == null) continue;
                if (imgsComplete.ContainsKey(name)) continue;
                //new Thread (() => DownloadPic(imgMatch.Value, name+".jpg")).Start();
                

                UrlInfo urlInfo = new UrlInfo(imgMatch.Value, "wait", name + ".jpg");
                ////CrawUrlEvent(urlInfo);
                imgsComplete.Add(name, true);
                imgcount++;
                a++;
            }*/
            foreach (Match match in matches)
            {
                string linkUrl = match.Groups["url"].Value;
                linkUrl = FixUrl(linkUrl, url);

                if (ifSkip(linkUrl)) continue;

                Match linkUrlMatch = Regex.Match(linkUrl, urlParseRegex);
                string protocal = linkUrlMatch.Groups["protocal"].Value;
                string ns = Regex.Replace(linkUrl, protocal, "http");
                string s = Regex.Replace(linkUrl, protocal, "https");
                string f = Regex.Replace(linkUrl, protocal, "ftp");
                if(!linkUrl.EndsWith("/")) other = linkUrl + "/";
                else other = linkUrl.TrimEnd(new char[] { '/' });

                if (urls[ns]==null && urls[s]==null && urls[f]==null && urls[other]==null)
                { 
                    urls[linkUrl] = false; 
                    pending.Enqueue(linkUrl);
                }
            }
        }
    }
}